/**
 *
 */
package com.spider.silence.worker;

import com.spider.silence.abstarct.downloader.AbstractCommonDownLoader;
import com.spider.silence.proxy.ProxyPool;
import com.spider.silence.task.MaLaTask;
import com.spider.silence.task.MalaModuleTask;
import com.spider.silence.task.manager.CommonTaskManager;
import com.spider.silence.utils.PropertiesUtil;
import org.apache.log4j.Logger;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import us.codecraft.webmagic.Page;

import java.net.URLDecoder;

/**
 * @author liyuanguo
 * @version V1.0
 * @title MalaModuleWorker.java
 * @time 2017年1月6日 上午10:29:15
 * @description
 */
public class MalaModuleWorker extends AbstractCommonDownLoader implements Runnable {
    private Logger logger = Logger.getLogger(MalaModuleWorker.class);

    @Override
    public void run() {
        while (true && !Thread.interrupted()) {
            try {
                MalaModuleTask moduleTask = (MalaModuleTask) CommonTaskManager.takeTask("malaModule", 1000);
                if (null != moduleTask) {
                    if (PropertiesUtil.getVal("MaLa.crawl.all").equals("true")) {
                        String moduleUrl = moduleTask.getModuleUrl();
                        Document first = get(moduleUrl);
                        String pageAll = first.select("#fd_page_top > div > label > span").text();
                        int total = Integer.parseInt(pageAll.substring(2, pageAll.length() - 2));
                        for (int pageNo = 1; pageNo < total + 1; pageNo++) {
                            String url = moduleUrl.substring(0, moduleUrl.length() - 6) + pageNo+".html";
                            boolean flag = true;
                            long startTime = System.currentTimeMillis();
                            long nowTime = startTime;
                            while (flag && nowTime - startTime < Long.valueOf(PropertiesUtil.getVal("MaLa.TimeOut"))) {
                                try {
                                    //doc= get("http://www.mala.cn/forum.php",true);
                                    Document document = null;
                                    Page pageContent = null;
                                    pageContent = getHtml(url,true);
                                    if (null == pageContent) {

                                    } else if (pageContent.getStatusCode() == 0) {

                                    } else if (String.valueOf(pageContent.getStatusCode()).startsWith("1")) {

                                    } else if (pageContent.getStatusCode() == 200) {
                                        flag = false;
                                        document = pageContent.getHtml().getDocument();
                                        //获取每个模块下的第一页所有帖子的url、组装任务
                                        Elements elements = null;
                                        if (pageNo==1) {
                                            elements = document.select("tbody").select("th").select("a.xst");
                                        }else{
                                            elements = document.select("tbody[id^='normalthread_'] a.xst");
                                        }
                                        for (int j = 0; j < elements.size(); j++) {
                                            String forumUrl = elements.get(j).attr("href");
                                            MaLaTask task = new MaLaTask(URLDecoder.decode(forumUrl, "UTF-8"));
                                            CommonTaskManager.add("mala", task);
                                        }
                                    } else if (String.valueOf(pageContent.getStatusCode()).startsWith("3")) {

                                    } else if (String.valueOf(pageContent.getStatusCode()).startsWith("4")) {

                                    } else if (String.valueOf(pageContent.getStatusCode()).startsWith("5")) {

                                    } else {

                                    }

                                } catch (Exception e1) {
                                    //System.out.println("异常信息："+e1.getMessage());
                                    logger.error(e1.getMessage(), e1);
                                } finally {
                                    nowTime = System.currentTimeMillis();
                                }
                            }


                        }
                    } else {
                        String moduleUrl = moduleTask.getModuleUrl();
                        int moduleNum = moduleTask.getModuleNum();
                        int moduleIndex = moduleTask.getModuleindex();
                        //Document document = get(moduleUrl,true);
                        boolean flag = true;
                        long startTime = System.currentTimeMillis();
                        long nowTime = startTime;
                        while (flag && nowTime - startTime < Long.valueOf(PropertiesUtil.getVal("MaLa.TimeOut"))) {
                            try {
                                //doc= get("http://www.mala.cn/forum.php",true);
                                Document document = null;
                                Page pageContent = null;
                                String getDataUrl = "";
                                getDataUrl = moduleUrl;
                                pageContent = getHtml(getDataUrl, true);
                                if (null == pageContent) {

                                } else if (pageContent.getStatusCode() == 0) {

                                } else if (String.valueOf(pageContent.getStatusCode()).startsWith("1")) {

                                } else if (pageContent.getStatusCode() == 200) {
                                    flag = false;
                                    //System.out.println("爬取模块下帖子url:"+moduleUrl+"状态码："+pageContent.getStatusCode());
                                    document = pageContent.getHtml().getDocument();
                                /*if(i==1&&page>1){
                                    Elements elementsPageUrl=document.select("#fd_page_bottom").select("a");
									if(null!=elementsPageUrl&&elementsPageUrl.size()>0){
										moduleUrl=elementsPageUrl.get(0).attr("href");
									}
								}*/

                                    //获取每个模块下的第一页所有帖子的url、组装任务
                                    Elements elements = null;
                                    if (moduleIndex < moduleNum - 1) {
                                        elements = document.select("tbody[id^='normalthread_'] a.xst");
                                    } else if (moduleIndex == (moduleNum - 1)) {
                                        elements = document.select("tbody").select("th").select("a.xst");
                                    }
                                    for (int j = 0; j < elements.size(); j++) {
                                        //String forumUrl="http://www.mala.cn/thread-14202544-1-1.html";
                                        String forumUrl = elements.get(j).attr("href");
                                        //System.out.println("forumUrl:"+forumUrl);
                                        MaLaTask task = new MaLaTask(URLDecoder.decode(forumUrl, "UTF-8"));
                                        CommonTaskManager.add("mala", task);
                                    }

                                } else if (String.valueOf(pageContent.getStatusCode()).startsWith("3")) {

                                } else if (String.valueOf(pageContent.getStatusCode()).startsWith("4")) {

                                } else if (String.valueOf(pageContent.getStatusCode()).startsWith("5")) {

                                } else {

                                }

                            } catch (Exception e1) {
                                //System.out.println("异常信息："+e1.getMessage());
                                logger.error(e1.getMessage(), e1);
                            } finally {
                                nowTime = System.currentTimeMillis();
                            }
                        }

                    }

                }

            } catch (Exception e) {
                logger.error(e.getMessage(),e);
                continue;
            }
        }

    }

}
